#!/bin/bash
#################################################################################################
# launch.sh:Lauch multiple dockers with AVD in it. 	   											#
# v0.1	2018/5/1	Wang Limin(Allen)															#
# v0.2	2018/5/16	Luo Yiwei	split launch emu and docker										#
# v0.3	2018/5/21	Luo Yiwei	support two GPUs												#
# v0.4	2018/5/22	Luo Yiwei	add ip forward check											#
# v0.5	2018/5/28	Luo Yiwei	add nfs support													#
# v0.6	2018/6/8	Luo Yiwei	add novnc and showkernel										#
# v0.7	2018/6/11	Luo Yiwei	update docker name with port									#
# v0.8	2018/7/3	Luo Yiwei	save emulator, ads, logcat std log;do not wait ads more			#
# v0.9	2018/7/10	Luo Yiwei	Use fixed port(15555) and device(5554) in docker				#
# v1.0	2018/7/17	Luo Yiwei	Add async parameter to support launch by group or parallel		#
# v1.1	2018/7/18	Luo Yiwei	Add kill emulator before stop docker							#
# v1.2	2018/7/20	Luo Yiwei	Add 4 cores support dor emulator								#
# v1.3	2018/7/24	Luo Yiwei	Fix -novnc parameter parse bug									#
# v1.4  2018/10/17	Yu Weiwei   support multiple 	GPUs.(more than 2)	
# v1.4  2018/11/22	jixng xin    add -sdkapi parameters while launching emulator  	            #
#################################################################################################
#WORK_PATH=/test_data/b107
#WORK_PATH=/workspace
#WORK_PATH=/data01/android-emulation-docker
WORK_PATH=/home/QemuB660/android-emulation-docker
NFS_LOCAL=/nfs
NFS_SHARE=0
export ANDROID_SDK_HOME=${WORK_PATH}/android-sdk-linux
export ANDROID_AVD_HOME=${ANDROID_SDK_HOME}/.android/avd
export ANDROID_SDK_ROOT=${ANDROID_SDK_HOME}
export ANDROID_SYS_DIR=${ANDROID_SDK_HOME}/system-images/android-24/default/arm64-v8a
export PATH="$PATH:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/platform-tools"
export ANDROID_ADB_PATH=/usr/bin/adb
# Please doucle check and make sure the files below exists!
dockerImage="ubuntu18.04cs_wc_on.tar"
#dockerImage="ubuntu18_mesa18.3.tar"
sdk_tar_name="android-sdk-linux.tgz"
gateway_ip=172.16.0.98
location=http://100.108.225.5/emulator
nfs_server=100.114.193.19:/nfs
# The name below is better not to be changed
docker_adb_path="${ANDROID_SDK_HOME}/platform-tools"
avd_name_prefix=avd
docker_xorg_name="xorg"
scriptDir=$(cd $(dirname $0); pwd)
scriptName=$(basename $0);
RED_COLOR='\E[1;31m'
GREEN_COLOR='\E[1;32m'
BLUE_COLOR='\E[1;34m'
COLOR_END='\E[0m'
novnc=0
#novnc=1
sdkapi=0
debug=0
showkernel=0
wipedata=0
async=0
mesa=0
userdefinedbridge=0

#################################################
#												#
# Print and execute a command					#
#												#
######################### ########################
logExec() { 
    local str_cmd="$*"
    echo -e "\n\$ ${str_cmd} \t\t@$(date "+%H:%M:%S")\n"
    ${str_cmd}
}

#################################################
#                								#
#  killProcess() 								#
#                								#
#################################################
killProcess(){
    local pids
    ulimit -c unlimited
    mkdir -p -m 777  /corefile
    echo "1" > /proc/sys/kernel/core_uses_pid
    echo "/corefile/core-%p-%s-%e-%t" > /proc/sys/kernel/core_pattern

    for i in $(ps aux|grep -v grep|grep launch|awk '{print $2}');do
    	kill -9 $i
    done
    killall -9 adb 2>/dev/null
    killall -9 ads 2>/dev/null 
    killall -9 qemu-system-aarch64 2>/dev/null
    killall -9 e2fsck 2>/dev/null
    killall -9 resize2fs 2>/dev/null
    killall -9 Xorg 2>/dev/null

    while true
    do
        pids=$(ps -ef|grep -v grep|grep -Ew '(ads|adb|avd|qemu-system-aarch64|e2fsck|resize2fs)'|awk '{print $2}')
        if [ "${pids}" ];then
            kill -9 ${pids} 2>/dev/null
            sleep 1
        else
    		break
        fi
    done
}


#################################################
#                								#
#  killDockers() 								#
#                								#
#################################################
killDockers()
{
    local dockers
    dockers=$(docker ps | grep ${docker_xorg_name} | awk '{print $NF}')
    for i in ${dockers}
    do
    	logExec "docker stop $i " &
    done
    dockers=$(docker ps |grep emulator-|awk '{print $NF}')
    for i in ${dockers}
    do
    	logExec "docker stop $i " &
    done
    wait
    dockers=$(docker ps -a | grep ${docker_xorg_name} | awk '{print $NF}')
    for i in ${dockers}
    do
    	logExec "docker rm $i " &
    done
    dockers=$(docker ps -a | grep emulator- | awk '{print $NF}')
    for i in ${dockers}
    do
    	logExec "docker rm $i " &
    done
    wait
    echo "Killing dockers is done!"
}

#################################################
#                								#
#  startX() launch xorg on X server docker		#
#                								#
#################################################
startX(){
    local docker_name=$1
    local host_tmp=$2
    logExec "rm -rf ${host_tmp}/.X*-lock ${host_tmp}/.X11-unix/X*"
    local opt
    local str_cmd
    opt=" --name ${docker_name}"
    opt+=" --device=/dev/tty0"
    opt+=" --device=/dev/vga_arbiter"
    if [ $(checkOSType) = "EulerOS" ];then
    	opt+=" --device=/dev/dri"
    else
    	opt+=" --device=/dev/fb0"
    fi
    opt+=" -v ${host_tmp}:/tmp" # /tmp path is shared between X server and X client
    opt+=" -v /var/run/dbus:/var/run/dbus"
    opt+=" --cap-add=ALL"
    opt+=" --security-opt=seccomp=unconfined"
    if [ ${mesa} -eq 1 ];then
        str_cmd="Xorg :0 -config /etc/X11/xrdp/xorg.conf"
    else
        str_cmd="Xorg :0 -config /tmp/xorg.conf -sharevts vt0"
    fi
    logExec "docker run -ti -d $opt ${dockerImage%*.tar}:latest ${str_cmd}"
    #logExec "docker run -ti -d $opt ${dockerImage%*.tar}:latest bash"
    docker exec -i ${docker_name} ps -ef|grep 'Xorg :0'|grep -vw grep
    while [ $? -ne 0 ];do
    	echo "Warning: Xorg process is not running... wait another second!"
    	sleep 1
    done
    logExec "iptables -F"
}

#############################
#                
#  checkOSType() # check the current OS type:Ubuntu, EulerOS or other Linux distribution
#                
#############################
checkOSType()
{
    if [ -f "/etc/lsb-release" ];then
        grep "Ubuntu" /etc/lsb-release > /dev/null
        if [ $? -eq 0 ];then
            echo "Ubuntu"
        else
            echo "Other"
        fi
    elif [ -f "/etc/euleros-release" ];then
        grep "EulerOS" /etc/euleros-release > /dev/null
        if [  $? -eq 0 ];then
            echo "EulerOS"
        else
    		echo "Other"
        fi
    fi
}

#################################################
#                								#
#  checkGPUNum() check GPU number 				#
#                								#
#################################################
checkGPUNum()
{
    if [ ${mesa} -eq 1 ];then
        local GPU_num=2
    else
        #local GPU_num=$(lspci|grep "VGA compatible controller"|wc -l)
        local GPU_num=2
    fi
    return $((GPU_num-1))
}

#################################################
#                								#
#  checkXorgConfig() check GPU number 				#
#                								#
#################################################
#checkXorgConfig()
#{
#	if [ ${GPU_num} -eq 1 ];then
#		if [ -e ${WORK_PATH}/xorg.conf ] && [ ! -e /tmp/xorg.conf ];then
#			cp ${WORK_PATH}/xorg.conf /tmp/xorg.conf
#		fi
#		if [ ! -e "/tmp/xorg.conf" ];then
#			echo "Error: the file /tmp/xorg.conf is not found!"
#			exit 1
#		fi
#	elif [ ${GPU_num} -eq 2 ];then
#		if [ ! -d /tmp_0 ];then
#			mkdir /tmp_0
#		fi
#		if [ ! -d /tmp_1 ];then
#			mkdir /tmp_1
#		fi
#		if [ -e ${WORK_PATH}/xorg0.conf ] && [ ! -e /tmp_0/xorg.conf ];then
#			cp ${WORK_PATH}/xorg0.conf /tmp_0/xorg.conf
#		fi
#		if [ -e ${WORK_PATH}/xorg1.conf ] && [ ! -e /tmp_1/xorg.conf ];then
#			cp ${WORK_PATH}/xorg1.conf /tmp_1/xorg.conf
#		fi
#		if [ ! -e "/tmp_0/xorg.conf" ];then
#			echo "Error: the file /tmp_0/xorg.conf is not found!"
#			exit 1
#		fi
#		if [ ! -e "/tmp_1/xorg.conf" ];then
#			echo "Error: the file /tmp_1/xorg.conf is not found!"
#			exit 1
#		fi
#	fi
#}

checkXorgConfig()
{
        checkGPUNum
        GPU_num=$?
        echo "GPU num is ${GPU_num} now"
        local loopnum=0
        while [ ${loopnum} -lt $GPU_num ]
        do
                if [ ! -d /tmp_${loopnum} ]; then
                        mkdir /tmp_${loopnum}
                fi
                if [ -e ${WORK_PATH}/xorg${loopnum}.conf ] && [ ! -e /tmp_${loopnum}/xorg.conf ];then
                        cp ${WORK_PATH}/xorg${loopnum}.conf /tmp_${loopnum}/xorg.conf
                fi
                if [ ! -e "/tmp_${loopnum}/xorg.conf" ];then
                        echo "Error: the file /tmp_${loopnum}/xorg.conf is not found!"
                        exit 1
                fi
		loopnum=$(($loopnum+1))
	done
}

#############################
#                
#  deploy() # install system
#                
#############################
deploy()
{
    local os=$1
    local system=system-images-$os
    cd ${scriptDir}
    if [ $? -ne 0 ];then
    	echo -e "${RED_COLOR} Error: failed to change directory to ${scriptDir}! ${COLOR_END}"
    	exit 1
    fi

    if [ $(checkOSType) = "EulerOS" ];then
    	# install docker and xserver components
    	logExec "yum install -y java expect docker-engine docker-runc docker-containerd xorg-x11-server-Xorg xorg-x11-drv-fbdev tigervnc-server nfs-utils samba-client subversion cifs-utils bridge-utils"
    	if [ $? -ne 0 ];then
    		echo -e "${RED_COLOR} Error: failed to install the required rpm packages. ${COLOR_END}"
    		exit 1
    	fi
    elif [ $(checkOSType) = "Ubuntu" ];then
    	logExec "apt-get install xubuntu-desktop xrdp docker.io openjdk-8-jdk -y"
    else
    	echo "Error: Unsupported Linux Distribution!"
    	exit 1
    fi

    if [ ${NFS_SHARE} -eq 0 ];then
    	if [ ! -d ${ANDROID_SDK_HOME} ] && [ ! -f ${sdk_tar_name} ];then
    		logExec "wget ${location}/${sdk_tar_name}"
    		if [ $? -ne 0 ];then
    			echo -e "${RED_COLOR} Error: failed to dowload the required sdk tarball. ${COLOR_END}"
    			exit 1
    		fi
    	fi

    	if [ ! -d ${system} ] && [ ! -f ${system}.tgz ];then
    		logExec "wget ${location}/${system}.tgz"
    		if [ $? -ne 0 ];then
    			echo -e "${RED_COLOR} Error: failed to dowload the required system image tarball. ${COLOR_END}"
    			exit 1
    		fi
    	fi

    	if [ ! -f ${dockerImage} ];then
    		logExec "wget ${location}/${dockerImage}"
    		if [ $? -ne 0 ];then
    			echo -e "${RED_COLOR} Error: failed to dowload the required docker image tarball. ${COLOR_END}"
    			exit 1
    		fi
    	fi

    	# extract sdk/system
    	if [ ! -d ${ANDROID_SDK_HOME} ];then
    		if [ "${sdk_tar_name##*.}" == "tgz" ] || [ "${sdk_tar_name##*.}" == "gz" ];then
    			logExec "tar -zxf ${sdk_tar_name}"
    			if [ $? -ne 0 ];then
    				echo -e "${RED_COLOR} Error: failed to extract the sdk tarball. ${COLOR_END}"
    				exit 1
    			fi
    		elif [ "${sdk_tar_name##*.}" == "tar" ];then
    			logExec "tar -xf ${sdk_tar_name}"
    			if [ $? -ne 0 ];then
    				echo -e "${RED_COLOR} Error: failed to extract the sdk tarball. ${COLOR_END}"
    				exit 1
    			fi
    		else
    			echo -e "${RED_COLOR} Error: unknown sdk tarball file type. ${COLOR_END}"
    			exit 1
    		fi
    	fi
    	if [ ! -d ${system} ];then
    		logExec "tar -zxf ${system}.tgz"
    		if [ $? -ne 0 ];then
    			echo -e "${RED_COLOR} Error: failed to extract the system image tarball. ${COLOR_END}"
    			exit 1
    		fi
    	fi 
    else # NFS_SHARE method
    	if [ ! -d "${NFS_LOCAL}" ];then
    		mkdir -p ${NFS_LOCAL}
    	fi
    	if [ ! -d ${WORK_PATH}/android-sdk-linux ];then
    		mkdir ${WORK_PATH}/android-sdk-linux
    	fi
    	if [ ! "$(mount | grep ${nfs_server})" ];then
    		mount -t nfs ${nfs_server} ${NFS_LOCAL}
    		if [ $? -ne 0 ];then
    			echo -e "${RED_COLOR} Error: failed to mount the remote nfs path ${nfs_server}. ${COLOR_END}"
    			exit 1
    		fi
    	fi
    	# extract sdk/system
    	if [ ! -d ${ANDROID_SDK_HOME} ];then
    		if [ "${sdk_tar_name##*.}" == "tgz" ] || [ "${sdk_tar_name##*.}" == "gz" ];then
    			logExec "tar -C ${NFS_LOCAL} -zxf ${NFS_LOCAL}/${sdk_tar_name}"
    			if [ $? -ne 0 ];then
    				echo -e "${RED_COLOR} Error: failed to extract the sdk tarball. ${COLOR_END}"
    				exit 1
    			fi
    		elif [ "${sdk_tar_name##*.}" == "tar" ];then
    			logExec "tar -C ${NFS_LOCAL} -xf ${NFS_LOCAL}/${sdk_tar_name}"
    			if [ $? -ne 0 ];then
    				echo -e "${RED_COLOR} Error: failed to extract the sdk tarball. ${COLOR_END}"
    				exit 1
    			fi
    		else
    			echo -e "${RED_COLOR} Error: unknown sdk tarball file type. ${COLOR_END}"
    			exit 1
    		fi
    	fi
    	if [ ! -d ${NFS_LOCAL}/${system} ];then
    		logExec "tar -C ${NFS_LOCAL} -zxf ${NFS_LOCAL}/${system}.tgz"
    		if [ $? -ne 0 ];then
    			echo -e "${RED_COLOR} Error: failed to extract the system image tarball. ${COLOR_END}"
    			exit 1
    		fi
    	fi 
    fi
    # locate system-images
    if [ ${NFS_SHARE} -eq 0 ];then
    	logExec "pushd ${ANDROID_SDK_HOME}"
    else
    	logExec "pushd ${NFS_LOCAL}/android-sdk-linux"
    fi
    
    ### check if the gles_mesa is renamed
    #if [ -d tools/lib64/gles_mesa ];then
    #	mv tools/lib64/gles_mesa tools/lib64/backup_gles_mesa_$(date "+%H-%M-%S")
    #fi
    if [ -d system-images ];then
    	mv system-images system-images_backup_$(date "+%H-%M-%S")
    fi
    if [ -L system-images ];then 
        unlink system-images
    fi
    logExec "ln -sf ../${system} system-images"

    # locate docker dockerImage
    if [ -L ${dockerImage} ];then 
        unlink ${dockerImage}
    fi
    logExec "ln -sf ../${dockerImage}"
    if [ -L ${ANDROID_SDK_HOME} ];then 
        unlink ${ANDROID_SDK_HOME}
    fi
    if [ ! -d ${ANDROID_SDK_HOME} ];then 
        echo -e "${RED_COLOR} Error: the sdk path does not exist. ${COLOR_END}"
    	exit 1
    fi
    logExec "popd"
    if [ ${NFS_SHARE} -ne 0 ];then
    	logExec "pushd ${ANDROID_SDK_HOME}"
    	for file_name in $(ls ${NFS_LOCAL}/android-sdk-linux|grep -v Readme)
    	do
    		ln -sf ${NFS_LOCAL}/android-sdk-linux/${file_name} ${file_name}
    	done
    	logExec "popd"
    fi
    # import docker images
    if [ ! "$(docker images -a|awk '{print $1}'|grep -w ${dockerImage%*.tar})" ];then
    	logExec "docker import ${ANDROID_SDK_HOME}/${dockerImage} ${dockerImage%*.tar}:latest"
    fi
}

#################################################
#                								#
#  checkReady() check readiness for emulator-$port	#
#                								#
#################################################
checkReady(){

    local i=$1
    local port=$((${i}*2+15555))
    local docker_name=emulator-${port}
    local device_name=emulator-5554
    local out_adb=$(docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} shell getprop sys.boot_completed 2>/dev/null | tail -n 1)
    while [ "${out_adb}" != "1" ];do
        sleep 5
    	if [ ${debug} -eq 1 ];then
    	echo "==== Info: wait for sys.boot_completed for emulator-${port}... ===="
    	fi
    	out_adb=$(docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} shell getprop sys.boot_completed 2>/dev/null | tail -n 1)
    done

    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} wait-for-device"
    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} remount"
    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} shell settings put secure adb_install_need_confirm 0"
}

#############################
#                
#  emuSet(),preset emulator-$port
#                
#############################
emuSet(){

    local i=$1
    local port=$(($i*2+15555))
    local docker_name=emulator-${port}
    local device_name=emulator-5554
#    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} forward tcp:15555 tcp:5555"
    sleep 1
    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} shell settings put GLOBAL device_provisioned 1"
    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} shell settings put SECURE user_setup_complete 1"
    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} shell settings put system screen_off_timeout 2147483647"
    logExec "docker exec -i ${docker_name} ${docker_adb_path}/adb -s ${device_name} shell settings put secure adb_install_need_confirm 0"
}

#############################
#
# removeAvd()
#
#############################

removeAvd()
{
    echo "========= start to remove all avds ========="
    rm -rf ${ANDROID_AVD_HOME}/${avd_name_prefix}_*.avd
    echo "========= removing all avds is done ========="
}

#############################
#
# createAvd() 
#
#############################

createAvd()
{
    local AVD_NAME="$1"
    local AVD_DIR="$2" 

    if [ -e ${AVD_DIR} ]; then
        # if avd exists, then delete it
    	echo -e "${BLUE_COLOR} ====================== Clean old existing avd ${AVD_NAME} ====================== ${COLOR_END}"
        rm -rf ${AVD_DIR}
    fi
    
    local RES=$(cat ${ANDROID_SYS_DIR}/out.config | awk -F= '{print $2}'|awk -F, '{print $1}'|sed 's#\(\s*\)##g')
    local DENSITY=$(cat ${ANDROID_SYS_DIR}/out.config | awk '{print $NF}')
	if [ ${sdkapi} -eq 0 ];then
		local SDK_API=$(cat ${ANDROID_SYS_DIR}/source.properties | grep AndroidVersion.ApiLevel | cut -d '=' -f2)
	else 
		SDK_API=${sdkapi}
	fi
   
    echo -e "${BLUE_COLOR} ========================= Start to create avd ${AVD_NAME} ========================= ${COLOR_END}"
    echo -e "${GREEN_COLOR}create emulator ${AVD_NAME} @res=${RES},dpi=${DENSITY}. ${COLOR_END} \t\t@$(date "+%H:%M:%S")"
    android - create avd -n ${AVD_NAME} -f -d "Nexus 6" -p ${AVD_DIR} -t android-${SDK_API} -b default/arm64-v8a -s ${RES} -c 4G
    if [ $? -ne 0 ];then
    	echo "Error: failed to create avd for ${AVD_NAME}. Exiting ..."
    	exit 1
    fi
    # set Internal storage
    sed -i '2a disk.dataPartition.size=1G' ${AVD_DIR}/config.ini
    sed -i '6a hw.camera.back=emulated'     ${AVD_DIR}/config.ini
    sed -i '7a hw.camera.front=emulated'    ${AVD_DIR}/config.ini
    sed -i 's/hw.keyboard=.*/hw.keyboard=yes/'            ${AVD_DIR}/config.ini

    #set Memory Options
    sed -i '17a hw.ramSize=1024'            ${AVD_DIR}/config.ini
    #sed -i '17a hw.ramSize=4096'            ${AVD_DIR}/config.ini
    sed -i '24a skin.dynamic=no'            ${AVD_DIR}/config.ini

    #set VM Heap
    sed -i '29a vm.heapSize=512'            ${AVD_DIR}/config.ini
    sed -i "s/hw.lcd.density=560/hw.lcd.density=${DENSITY}/" ${AVD_DIR}/config.ini


    rm -f ${AVD_DIR}/userdata.img
    ln -fs ${ANDROID_SYS_DIR}/userdata.img ${AVD_DIR}/userdata.img

    rm -f ${AVD_DIR}/system.img
    ln -fs ${ANDROID_SYS_DIR}/system.img   ${AVD_DIR}/system.img

    if [ -f ${ANDROID_SYS_DIR}/vendor.img ];then
        rm -f ${AVD_DIR}/vendor.img
        ln -fs ${ANDROID_SYS_DIR}/vendor.img   ${AVD_DIR}/vendor.img
    fi

    echo -e "${GREEN_COLOR} Creation of avd ${AVD_NAME} is finished! ${COLOR_END} \t\t@$(date "+%H:%M:%S")"
}

########################################
#
# createAvdMN() :  create avd_m:n
#
########################################
createAvdMN()
{
    local range=$*
    local port_num=0
    for i in $range
    do
    	port_num=$(($i*2+15555))
        createAvd ${avd_name_prefix}_${port_num} ${ANDROID_AVD_HOME}/${avd_name_prefix}_${port_num}.avd &
    done
    wait
}

#############################
#
# check_ip_forward()
#
#############################
check_ip_forward()
{
    local rv=$(sysctl net.ipv4.ip_forward|awk '{print $NF}')
    if [ ${rv} -eq 0 ];then
    	sed -i '/net.ipv4.ip_forward/ s/\(.*=\).*/\11/' /etc/sysctl.conf
    	echo -e "${RED_COLOR} The default ipv4 forwarding is disabled on this host. The network service will be restarted in 3 seconds, maybe you need to execute launch -d <name> again, please pay attention! ${COLOR_END}"
    	sleep 3
    	if [ $(checkOSType) = "EulerOS" ];then
    		systemctl restart network.service
    	elif [ $(checkOSType) = "Ubuntu" ];then
    		/etc/init.d/networking restart
    	fi
    fi
}

add_proxy()
{
cat <<EOF > /etc/sysconfig/network
NETWORKING=yes
NETWORKING_IPV6=yes
GATEWAY=${gateway_ip}
EOF

cat <<EOF > /etc/sysconfig/network-scripts/route-bond0
10.0.0.0/8 via 172.16.0.1
100.0.0.0/8 via 172.16.0.1
EOF

    if [ $(checkOSType) = "EulerOS" ];then
    	systemctl restart network.service
    elif [ $(checkOSType) = "Ubuntu" ];then
    	/etc/init.d/networking restart
    fi
}
#############################
#
# setEnv()
#
#############################
setEnv()
{
	if [ $(checkOSType) = "EulerOS" ];then
		check_ip_forward
		#add_proxy
	fi
	if [ ! -d /corefile ];then
		mkdir -p -m 777  /corefile
	fi
    local os=${1:-euler}
    setNetBridge
    deploy $os
    #killall -9 qemu-system-aarch64 2>/dev/null
    if [ -e /proc/sys/kernel/numa_balancing ];then
    	echo '0' > /proc/sys/kernel/numa_balancing
    fi
}
#############################
#
#setNetBridge()
#
#############################
setNetBridge()
{
#local dnat_index=$(iptables -n -L PREROUTING -t nat --line-numbers | grep DNAT | grep 192.168.5 | awk '{print $1}')
#local j
#for j in ${dnat_index}
#do
#    sudo iptables -t nat -D PREROUTING $j
#done
#local snat_index=$(iptables -n -L POSTROUTING -t nat --line-numbers | grep SNAT | grep 192.168.5 | awk '{print $1}')
#local k
#for k in ${snat_index}
#do
#    sudo iptables -t nat -D POSTROUTING $k
#done
iptables -t nat -F

if [ ${userdefinedbridge} -eq 1 ];then
    echo "use user-defined bridge0!!!"
    service docker stop
    ifconfig docker0 > /dev/null
    if [ $? -eq 0 ];then
       echo "delete default bridge docker0!!"
       logExec "ip link set dev docker0 down"
       logExec "brctl delbr docker0"
       ifconfig > /root/2
    fi

    ifconfig bridge0 > /dev/null
    if [ $? -ne 0 ];then
        echo "add bridge0!!!"
        brctl addbr bridge0
        ip addr add 192.168.5.1/24 dev bridge0
        ip link set dev bridge0 up
    fi

    local n=`grep -n "OPTIONS" /etc/sysconfig/docker | awk -F: '{print $1}'`
    TMP="OPTIONS='--live-restore -b=bridge0'"
    sed -i "$[ n ]c $TMP" /etc/sysconfig/docker
    service docker start
    
    if [ $? -eq 0 ];then
       echo "delete default bridge docker0!!"
       logExec "ip link set dev docker0 down"
       logExec "brctl delbr docker0"
       ifconfig > /root/2
    fi
else
    echo "use default bridge docker0!!!"
    service docker stop
    ifconfig bridge0 > /dev/null
    if [ $? -eq 0 ];then
        echo "delete bridge0!!!"
        ip link set dev bridge0 down
        brctl delbr bridge0
    fi
    local n=`grep -n "OPTIONS" /etc/sysconfig/docker | awk -F: '{print $1}'`
    TMP="OPTIONS='--live-restore'"
    sed -i "$[ n ]c $TMP" /etc/sysconfig/docker
    service docker start
fi
}
#############################
#
# upgradeEnv()
#
#############################
upgradeEnv()
{
    local range=$*
    echo "====================== remove existing sdk path ==============================="
    rm -rf ${ANDROID_SDK_HOME}
    echo "=================== stop and remove running dockers ==========================="
    killDockers
    sleep 3
    echo "============ remove old docker images ${dockerImage%*.tar} ===================="
    if [ "$(docker images |grep ${dockerImage%*.tar})" ];then
    	docker rmi ${dockerImage%*.tar}
    fi
    setEnv ${range}
}

#################################################
#												#
# clean()										#
#												#
#################################################
clean()
{
    killDockers
    killProcess 
}

dockerClean()
{
    if [ $(checkOSType) = "EulerOS" ];then
    	systemctl stop docker
    elif [ $(checkOSType) = "Ubuntu" ];then
    	service docker stop
    fi
    for x in $(mount|grep docker|awk '{print $3}');do umount $x; done
    rm -rf /var/lib/docker
    mount|grep docker
    if [ $(checkOSType) = "EulerOS" ];then
    	systemctl start docker
    elif [ $(checkOSType) = "Ubuntu" ];then
    	service docker start
    fi
}

##########################################################
#                
#  launchDocker() # lauchDocker emulator_i
#                
##########################################################
launchDocker(){

    local i=$1
    local host_tmp=$2
    local port=$(($i*2+15555))
    local vnc=$(($i+6901))
    local ads_port=$(($i*2+6554))
    local opt=" --name=emulator-${port}"
    opt+=" -v ${WORK_PATH}:${WORK_PATH}"
    if [ ${NFS_SHARE} -ne 0 ];then
    	opt+=" -v ${NFS_LOCAL}:${NFS_LOCAL}"
    fi
    opt+=" -v /corefile:/corefile"
    opt+=" -v /usr/share/X11/xkb:/usr/share/X11/xkb"
    opt+=" -v /var/run/dbus:/var/run/dbus"
    opt+=" -v ${host_tmp}:/tmp"
    #opt+=" -v /tmp_0:/tmp"
    if [ ${userdefinedbridge} -eq 0 ];then
        opt+=" -p $port:15555"
        opt+=" -p $vnc:$vnc"
#        opt+=" -p ${ads_port}:6554"
    fi
    opt+=" -e ANDROID_SDK_HOME=${ANDROID_SDK_HOME}"
    opt+=" -e ANDROID_AVD_HOME=${ANDROID_AVD_HOME}"
    opt+=" -e DISPLAY=:0"
    opt+=" -e port=15555"
#	opt+=" --net=bridge"
    opt+=" --device=/dev/kvm"
    opt+=" --device=/dev/bus/usb"
    if [ $(checkOSType) = "EulerOS" ];then
    	opt+=" --device=/dev/dri"
    else
    	opt+=" --device=/dev/fb0"
    fi
    opt+=" --device=/dev/vga_arbiter"
    opt+=" --device=/dev/fb0"
    opt+=" --device=/dev/tty0"
    opt+=" --security-opt seccomp=unconfined"
    logExec "docker run -ti -d $opt ${dockerImage%*.tar}:latest bash"
    if [ $? -ne 0 ];then
    	sleep 5
    	echo "======== Error: failed to start emulator-${port} ========"
    	docker start --cpuset-cpus=3,4 --cpuset-mems=0  emulator-${port}
    fi
    sleep 15
}

#################################################
#												#
# docLaunchMN() stub func for emulator-m:n		#
#												#
#################################################
docLaunchMN()
{
    local range=$*
    echo -e "${BLUE_COLOR} ======================== Start to Launch Docker for Emulator ===================== ${COLOR_END}"
    for i in ${range}
    do
#		if [ ${GPU_num} -le 1 ];then
#			launchDocker $i "/tmp" &
#		else
#			launchDocker $i "/tmp_$(($i%2))" &
#		fi
		launchDocker $i "/tmp_$(($i%${GPU_num}))" &
    done
    wait
}

#################################################
#												#
# launchEmubyGroup() stub func for emulator-m:n#
#												#
#################################################
launchEmubyGroup()
{
    local range=$*
    local range_array=(${range})
    local scale=10
    local loop_times=0
    local i=0
    local j=0
    emuCount=$(echo ${range} | wc -w)
    if [ ${emuCount} -ge ${scale} ];then
    	loop_times=$((${emuCount}/${scale}))
    	while [ ${i} -lt ${loop_times} ];do
    		j=0
    		while [ ${j} -lt ${scale} ];do
    			launchEmu ${range_array[$((${i}*${scale}+${j}))]} &
    			((j++))
    		done
    		((i++))
    		echo -e "${BLUE_COLOR} ============ The ${i}th group of emulator is launching ============= ${COLOR_END}"
    		wait
    	done
    fi
    loop_times=$((${emuCount}%${scale}))
    j=0
    while [ ${j} -lt ${loop_times} ];do
    	launchEmu ${range_array[$((${i}*${scale}+${j}))]} &
    	((j++))
    done
    echo -e "${BLUE_COLOR} =================== The odd part emulator is launching ==================== ${COLOR_END}"
    wait
}

#################################################
#												#
# launchEmuParallel() stub func for emulator-m:n#
#												#
#################################################
launchEmuParallel()
{
    local range=$*
    for i in $range
    do
    	launchEmu $i &
    done
    wait
}

#################################################
#												#
# emuLaunchMN() stub func for emulator-m:n		#
#												#
#################################################
emuLaunchMN()
{
    local range=$*
    echo -e "${BLUE_COLOR} ====================== Start to Launch Emulator within Docker ====================== ${COLOR_END}"
    if [ ${async} -eq 0 ];then
    	launchEmubyGroup ${range} &
    else
    	launchEmuParallel ${range} &
    fi
    wait
    echo -e "${BLUE_COLOR} ======================== Start to Check Ready for Emulator ======================== ${COLOR_END}"
    for i in $range
    do
    	checkReady $i &
    done
    wait
    echo -e "${BLUE_COLOR} =========================== Wait Emulator Lauch Finished =========================== ${COLOR_END}"
    emuSetMN $range
    echo -e "${BLUE_COLOR} ================== emuSet and Launching is Finished =================== @$(date "+%H:%M:%S") ${COLOR_END}"
    endTime=$(date +%s)
    costTime=$((${endTime}-${startTime}))
    echo -e "${BLUE_COLOR} ================== Start@${beginTime} End@$(date "+%H:%M:%S") Total Time ${costTime}S ===================${COLOR_END}"
    echo "Start@${beginTime},End@$(date "+%H:%M:%S"),Total_Time:${costTime}" >> /root/time_statistics.txt
}

#################################################
#												#
# saveLogcat										#
#												#
#################################################
saveLogcat()
{
    local i=$1
    local port=$(($i*2+15555))
    local log_file_name=$(date "+%H-%M-%S")
    echo "docker exec -i emulator-${port} bash -c \"adb logcat >/tmp/logcat-${port}-${log_file_name}.log 2>&1\" &"
    docker exec -i emulator-${port} bash -c "adb logcat >/tmp/logcat-${port}-${log_file_name}.log 2>&1" &
}

#################################################
#												#
# startAds										#
#												#
#################################################
startAds()
{
    local i=$1
    local port=$(($i*2+15555))
    local log_file_name=$(date "+%H-%M-%S")
    local ads_proc_num=$(ps aux|grep emulator-${port}|grep ads.jar|wc -l)
    if [ ${ads_proc_num} -eq 0 ];then
    	echo "docker exec -i emulator-${port} bash -c \"/usr/bin/java -jar /usr/bin/ads.jar ADB=/usr/bin/adb REBOOT_ON_CRASH=true >/tmp/adslog-${port}-${log_file_name}.log 2>&1\" &"
    	docker exec -i emulator-${port} bash -c "/usr/bin/java -jar /usr/bin/ads.jar ADB=/usr/bin/adb REBOOT_ON_CRASH=true >/tmp/adslog-${port}-${log_file_name}.log 2>&1" &
    else
    	echo -e "${GREEN_COLOR} ====== ads process has been running in emulator-${port}, skip launching it. ====== ${COLOR_END}"
    fi
    docker exec -i emulator-${port} bash -c "netstat -nlp |grep 6554"
    while [ -z "$(docker exec -i emulator-${port} bash -c 'netstat -nlp |grep 6554')" ];do
	echo "wait one second for listening on 6554 ready for emulator-${port}"
        sleep 1
    done
}

#################################################
#												#
# startAdsMN()									#
#												#
#################################################
startAdsMN()
{
    local range=$*
    for i in ${range}
    do
       startAds $i &
       #saveLogcat $i &
    done
    sleep 1
    wait
    echo -e "${BLUE_COLOR} ============================= ADS Started ============================= @$(date "+%H:%M:%S") ${COLOR_END}"
}

#################################################
#												#
# checkReadyMN()								#
#												#
#################################################
checkReadyMN()
{
    local range=$*
    startTime=$(date +%s)
    for i in ${range}
    do
       checkReady $i &
    done
    wait

    emuSetMN ${range}
}

#################################################
#												#
# cleanOldLogFiles()								#
#												#
#################################################
cleanOldLogFiles()
{
    local i=0
    for filename in $(ls -t /corefile)
    do
    	if [ $i -eq 0 ];then
    		i=$((i+1))
    	else
    		rm -rf /corefile/${filename}
    	fi
    done
    #find /tmp -mtime +30 -type f -name emulator*.log -exec rm -rf {} \;
    #find /tmp -mtime +30 -type f -name logcat*.log -exec rm -rf {} \;
    #find /tmp -mtime +30 -type f -name adslog*.log -exec rm -rf {} \;
#	if [ ${GPU_num} -le 1 ];then
#		find /tmp -type f -name emulator*.log -exec rm -rf {} \;
#		find /tmp -type f -name logcat*.log -exec rm -rf {} \;
#		find /tmp -type f -name adslog*.log -exec rm -rf {} \;
#	else
#		find /tmp_0 -type f -name emulator*.log -exec rm -rf {} \;
#		find /tmp_0 -type f -name logcat*.log -exec rm -rf {} \;
#		find /tmp_0 -type f -name adslog*.log -exec rm -rf {} \;
#		find /tmp_1 -type f -name emulator*.log -exec rm -rf {} \;
#		find /tmp_1 -type f -name logcat*.log -exec rm -rf {} \;
#		find /tmp_1 -type f -name adslog*.log -exec rm -rf {} \;
#	fi
        local loopnum=0
        while [ ${loopnum} -lt $GPU_num ]
	do
		find /tmp_${loopnum} -type f -name emulator*.log -exec rm -rf {} \;
                find /tmp_${loopnum} -type f -name logcat*.log -exec rm -rf {} \;
               	find /tmp_${loopnum} -type f -name adslog*.log -exec rm -rf {} \;	
		loopnum=$(($loopnum+1))
	done
}

########################################
#
# xorgStop() stub func for emulator-m:n
#
########################################
xorgStop()
{
	echo -e "${BLUE_COLOR} =========================== Stop Running Xorg Docker ======================== ${COLOR_END}"
#	if [ ${GPU_num} -le 1 ];then
#		if [ "$(docker ps | grep ${docker_xorg_name})" ];then
#		docker stop ${docker_xorg_name} &
#		fi
#	else
#		if [ "$(docker ps | grep ${docker_xorg_name}_0)" ];then
#			docker stop ${docker_xorg_name}_0 &
#		fi
#		if [ "$(docker ps | grep ${docker_xorg_name}_1)" ];then
#			docker stop ${docker_xorg_name}_1 &
#		fi
#	fi
        local loopnum=0
        while [ ${loopnum} -lt $GPU_num ]
	do
		if [ "$(docker ps | grep ${docker_xorg_name}_${loopnum})" ];then
			docker stop ${docker_xorg_name}_${loopnum} &
		fi
		loopnum=$(($loopnum+1))
	done	

	wait
#	if [ ${GPU_num} -le 1 ];then
#		if [ "$(docker ps -a | grep ${docker_xorg_name})" ];then
#		docker rm ${docker_xorg_name} &
#		fi
#	else
#		if [ "$(docker ps -a  | grep ${docker_xorg_name}_0)" ];then
#			docker rm ${docker_xorg_name}_0 &
#		fi
#		if [ "$(docker ps -a  | grep ${docker_xorg_name}_1)" ];then
#			docker rm ${docker_xorg_name}_1 &
#		fi
#	fi
	loopnum=0
        while [ ${loopnum} -lt $GPU_num ]
	do
		if [ "$(docker ps -a  | grep ${docker_xorg_name}_${loopnum})" ];then
			docker rm ${docker_xorg_name}_${loopnum} &
		fi
		loopnum=$(($loopnum+1))
	done

	wait
	echo "Stopping xorg dockers is done!"
}

########################################
#
# emuDockerStopMN() stub func for emulator-m:n
#
########################################
emuDockerStopMN()
{
    local range=$*
    local port=0
    killEmuMN ${range}
    echo -e "${BLUE_COLOR} =========================== Stop Running Emu Docker ======================== ${COLOR_END}"
    for i in ${range}
    do
    	port=$(($i*2+15555))
    	if [ "$(docker ps |grep emulator-${port})" ];then
    		docker stop emulator-${port} &
    	fi
    done
    wait

    for i in ${range}
    do
    	port=$(($i*2+15555))
    	if [ "$(docker ps -a|grep emulator-${port})" ];then
    		docker rm emulator-${port} &
    	fi
    done
    wait
    echo "Stopping emulator dockers is done!"
}
########################################
#
# stopMN() stub func for emulator-m:n
#
########################################
stopMN()
{
    local range=$*
    local port=0
    emuDockerStopMN ${range}
    xorgStop
}


########################################
#
# startMN() stub func for emulator-m:n
#
########################################
startMN()
{
	local range=$*
	cleanOldLogFiles
	startTime=$(date +%s)
	beginTime=$(date "+%H:%M:%S")
	echo -e "${BLUE_COLOR} ========================= Start to Launch Xorg Docker ====================== @$(date "+%H:%M:%S") ${COLOR_END}"
#	if [ ${GPU_num} -le 1 ] && [ ! "$(ps -ef|grep 'Xorg :0'|grep -vw grep)" ];then
#		startX ${docker_xorg_name} "/tmp"
#	elif [ ${GPU_num} -eq 2 ] && [ $(ps -ef|grep 'Xorg :0'|grep -vw grep|wc -l) -ne 2 ];then
#		killall -9 Xorg 2>/dev/null
#		startX "${docker_xorg_name}_0" "/tmp_0"
#		startX "${docker_xorg_name}_1" "/tmp_1"
#	fi

        if [ $(ps -ef|grep 'Xorg :0'|grep -vw grep|wc -l) -ne $GPU_num ];then
                killall -9 Xorg 2>/dev/null
        	local loopnum=0
        	while [ ${loopnum} -lt $GPU_num ]
		do
			startX "${docker_xorg_name}_${loopnum}" "/tmp_${loopnum}"
			loopnum=$(($loopnum+1))
		done
	fi
        #local loopnum=0
        #while [ ${loopnum} -lt $GPU_num ]
	#do
	#	startX "${docker_xorg_name}_${loopnum}" "/tmp_${loopnum}"
	#	loopnum=$(($loopnum+1))
	#done	
	
	docLaunchMN ${range}
	#startAdsMN ${range}
    emuLaunchMN ${range}
}  

#################################################
#												#
#relaunchEmuDocker() stub func for relaunch docker#
#												#
#################################################
relaunchEmuDocker()
{
    local range=$*
    local port=0
    emuDockerStopMN ${range}
    startTime=$(date +%s)
    docLaunchMN ${range}
       #startAdsMN ${range}
        emuLaunchMN ${range}
}
#################################################
#												#
# restartMN() stub func for emulator-m:n		#
#												#
#################################################
restartMN()
{
    local range=$*
	stopMN ${range}
    startMN ${range}
}

########################################
#
# restartMN_debug() stub func for emulator-m:n with debug
#
########################################
restartMN_debug()
{
    debug=1
    local range=$*
    stopMN ${range}
    startMN ${range}
}

########################################
#
# emuSetMN() preset emulator-m:n
#
########################################
emuSetMN()
{
    local range=$*
    for i in ${range}
    do
        emuSet $i &
    done
    wait
    echo -e "${BLUE_COLOR} ================== Start to Launch ADS Process =================== @$(date "+%H:%M:%S") ${COLOR_END}"
    
}

#################################################
#                								#
#  isReady() # 									#
#     check readiness for emulator_$i in docker	#
#     this function is not used currently		#
#                								#
#################################################

isReady(){

    local port=$1
    local vnc_port=$(($2+5900))
    local j=0
    local h=$(($port-10001))
    #local sleep_time=$(($(echo ${range} | wc -w)*5))
    local sleep_time=5
    local total_sleep_time=0
    logExec "docker exec -i emulator-${port} ${docker_adb_path}/adb -s emulator-5554 wait-for-device"
    echo -e "\n\$ docker exec -i emulator-${port} ${docker_adb_path}/adb shell getprop sys.boot_completed|tail -n 1 \t\t@$(date "+%H:%M:%S")\n"
    local out_adb=$(docker exec -i emulator-${port} ${docker_adb_path}/adb shell getprop sys.boot_completed 2>/dev/null)
    while [ "$(echo ${out_adb} |tail -n 1)" != "1" ];do
    	if [ ${total_sleep_time} -gt 3600 ];then
    		echo -e "${RED_COLOR} ================== TIME OUT FOR 3600S, EXITING for emulator-${port} ================== ${COLOR_END}"
    		echo "waiting for emulator-${port} timeout" >> /tmp/launch_test.log
    		exit 1
    	fi
    	sleep ${sleep_time}
    	let total_sleep_time=total_sleep_time+sleep_time
    	out_adb=$(docker exec -i emulator-${port} ${docker_adb_path}/adb shell getprop sys.boot_completed 2>/dev/null)
    	if [ ${debug} -eq 1 ];then
    		((j++))
    		echo "==== $j times ==== Info: wait for sys.boot_completed for emulator-${port}..."
    	fi
    done
    logExec "docker exec -i emulator-${port} ${docker_adb_path}/adb  forward tcp:15555 tcp:5555"

    if [ ${userdefinedbridge} -eq 1 ];then
        local host_ip=$(ifconfig eth0|grep inet|grep -v inet6|awk '{print $2}')
        local docker_ip=$(docker exec -i emulator-${port} ifconfig eth0|grep inet|grep -v inet6|awk '{print $2}')

        echo "host_ip=${host_ip},docker=${docker_ip},vnc_port=${vnc_port},port=${port}"
        iptables -n -L PREROUTING -t nat --line-numbers|grep DNAT|grep "tcp dpt:${port}"
        if [ $? -ne 0 ];then
           logExec "iptables -t nat -A PREROUTING -d ${host_ip} -p tcp --dport ${port} -j DNAT --to ${docker_ip}:15555"
        fi
        iptables -n -L PREROUTING -t nat --line-numbers|grep DNAT|grep "tcp dpt:${vnc_port}"
        if [ $? -ne 0 ];then
           logExec "iptables -t nat -A PREROUTING -d ${host_ip} -p tcp --dport ${vnc_port} -j DNAT --to ${docker_ip}:${vnc_port}"
        fi
        iptables -n -L POSTROUTING -t nat --line-numbers|grep SNAT|grep ${docker_ip}
        if [ $? -ne 0 ];then
           logExec "iptables -t nat -A POSTROUTING -s ${docker_ip}/32 -j SNAT --to ${host_ip}"
        fi
        sleep 1
    fi
}

#############################################################
#                										`	#
#  launchEmu() # lauch emulator in the specified dokcer	#
#                											#
#############################################################

launchEmu()
{
    local i=$1
    local nnuma=$(lscpu|grep '^NUMA node(s)'|awk '{print $NF}')
    local m=$(($i%$nnuma))
    local port=$(($i*2+5554))
    local port1=$(($port+1))
    local port2=$(($port+10001))
    local avd_name=${avd_name_prefix}_${port2}
    #local patched_kernel=${WORK_PATH}/kernel_4.4_patched
    #local patched_kernel=${WORK_PATH}/kernel_4.4_20190128
    #local patched_kernel=${WORK_PATH}/kernel_3.18_patched_alex_20190123
    local patched_kernel=${WORK_PATH}/kernel-ranchu
    local vnc_num=$(($i+1001))
    #local sleep_time=$(($(echo ${range} | wc -w)*10))
    local sleep_time=10
    local opt=""
    local log_file_name=$(date "+%H-%M-%S")
    #logExec "docker exec -i emulator-${port2} ${docker_adb_path}/adb start-server"
    sleep 1
    opt+=" -avd ${avd_name}"
    opt+=" -port 5554"
    opt+=" -kernel ${patched_kernel}"
    opt+=" -prop sys.prop.writeimei=868035034034600"
    opt+=" -writable-system"
    opt+=" -camera-back emulated"
    opt+=" -camera-front emulated"
    opt+="  -memory 4096"
    #opt+=" -memory 6144"
    #opt+=" -memory 3072"
    opt+=" -partition-size 1024"
    #opt+=" -partition-size 4096"
    if [ ${showkernel} -eq 1 ];then
    	opt+=" -show-kernel"
    fi
    if [ ${wipedata} -eq 1 ];then
    	opt+=" -wipe-data"
    fi
    if [ ${mesa} -eq 1 ];then
        opt+=" -gpu mesa"
    else
        opt+=" -gpu host"
    fi
    opt+=" -cores 2"
    #opt+=" -cores 1"
    opt+=" -no-snapshot"
    opt+=" -no-window"
    opt+=" -show-kernel"
    opt+=" -logcat *:*"
    opt+=" -engine qemu2"
    #opt+=" -mem-path /dev/hugepages"
    #opt+=" -mem-prealloc"
    #opt+=" -realtime min_guarantee=3145728,mlock=off"
    opt+=" -qemu --enable-kvm"
    if [ ${novnc} -eq 0 ];then
     	opt+=" -vnc :${vnc_num}"
    fi
    logDir=/home/leibinghan/log
    log=$logDir/emulator-${port2}-${log_file_name}.log
    touch $log
    chmod 777 $log
    #sleep 10
    logExec "docker exec -i emulator-${port2} rm -rf ${ANDROID_AVD_HOME}/${avd_name}.avd/hardware-qemu.ini.lock"
    echo "docker exec -i emulator-${port2} bash -c \"numactl -m $m -N $m ${ANDROID_SDK_HOME}/tools/emulator $opt >$log 2>&1\" &"
    #echo "docker exec -i emulator-${port2} bash -c \"numactl  ${ANDROID_SDK_HOME}/tools/emulator $opt >$log 2>&1\" &"
    
    #docker exec -i emulator-${port2} bash -c "numactl   --interleave=all --physcpubind 0,1  ${ANDROID_SDK_HOME}/tools/emulator $opt >$log 2>&1" &
    docker exec -i emulator-${port2} bash -c "numactl  -m $m -N $m  ${ANDROID_SDK_HOME}/tools/emulator $opt" > $log 2>&1 &
    #docker exec -i emulator-${port2} bash -c "numactl  -m $m -N $m  ${ANDROID_SDK_HOME}/tools/emulator $opt "  &
    #docker exec -i emulator-${port2} bash -c " ${ANDROID_SDK_HOME}/tools/emulator $opt >$log 2>&1" &
    
    #docker exec -i emulator-${port2} bash -c "echo startavd numactl -m $m -N $m ${ANDROID_SDK_HOME}/tools/emulator $opt | telnet 127.0.0.1 6554"
    echo -e "${GREEN_COLOR}Info: Sleep ${sleep_time} seconds to wait emulators-${port2} online! ${COLOR_END}"
    sleep ${sleep_time}
    isReady ${port2} ${vnc_num}
    echo -e "\n\$ docker exec -i emulator-${port2} netstat -nlp|grep -w 15555|grep -v grep \t\t@$(date "+%H:%M:%S")\n"
    local output_netstat=$(docker exec -i emulator-${port2} netstat -nlp)
    while [ ! "$(echo ${output_netstat}|grep -w 15555|grep -v grep)" ]
    do
    	logExec "docker exec -i emulator-${port2} ${docker_adb_path}/adb forward tcp:15555 tcp:5555"
    	sleep 1
    	output_netstat=$(docker exec -i emulator-${port2} netstat -nlp)
    done
}

##########################################################
#                
#  killEmuMN() # kill emulator process in docker M:N
#                
##########################################################
killEmuMN()
{
    local range=$*
    echo -e "${BLUE_COLOR} ========================= Start to Kill Existing Emulator ========================= ${COLOR_END}"
    for i in $range
    do
    	local port=$(($i*2+15555))
    	if [ "$(docker ps |grep emulator-${port})" ];then
    		pid_list=$(docker exec -i emulator-${port} ps aux|grep qemu|awk '{print $2}')
    		for pid in ${pid_list}
    		do
    			logExec "docker exec -i emulator-${port} kill -9 ${pid}" &
    		done
    	fi
    done
    wait
}

##########################################################
#                
#  relaunchEmuMN() # relaunch emulator M:N
#                
##########################################################
relaunchEmuMN() {
    local range=$*
    killEmuMN ${range}
    startTime=$(date +%s)
    emuLaunchMN ${range}
}

#################################################
#												#
#	range() generate sequence number by minus	#
#	and comma, 									#
#	e.g. 1,6,3-7,9-12 => 1 3 4 5 6 7 9 10 11 12	#
#												#
#################################################
rangeN()
{
    local str=$1
    for i in $(echo "$str"|sed 's#\s*##g'|sed 's#,# #g')
    do 
        if [ $(echo $i|grep -) ];then
    		local begin_num=${i%%-*}
    		local end_num=${i##*-}
    		if [ ${begin_num} -gt ${end_num} ];then
    			echo "Error: the begin number is greater than the end number!"
    			exit 1
    		fi
            while [ ${begin_num} -le ${end_num} ];do 
                echo "${begin_num}"
    			let begin_num++
            done
        else 
            echo "$i"
        fi
    done

}

########################################
#
# version() help page
#
########################################
version()
{
    echo 1.3
}

########################################
#
# _help() help page
#
########################################
_help()
{
    echo -e "
Usage: $0 [operation] [number of emu] [extra opt]

Operation and Emulator Range:
    -d <postfix name of system image>		deploy emulator system
    -u <postfix name of system image>		clean the current SDK amd docker image and then deploy emulator system
    -upgrade <postfix name of system image>		clean the current SDK amd docker image and then deploy emulator system
    -a  \$start-\$end					creaete avd from avd-m:n
    -r  \$start-\$end					restart the emulator-m:n
    -restart  \$start-\$end				restart the emulator-m:n
    -stop  \$start-\$end				stop the emulator-m:n
    -start  \$start-\$end				stop the emulator-m:n
    -rd  \$start-\$end				restart the emulator-m:n with more debug output information
    -s  \$start-\$end					only set status on running emulator-m:n
    -e  \$start-\$end					launch emulator-m:n on running emulator-m:n
    -relaunch  \$start-\$end				only relaunch the docker for specified emulator-m:n docker without xorg docker
    -chk  \$start-\$end				check the readiness for running emulator-m:n
    -c				 		clean all emulators
    -cc						umount all the mounted docker and restart docker service
    -h						print help information
Extra Options:
    -showkernel				add -show-kernel parameters while launching emulators
    -wipedata				add -wipe-data parameters while launching emulators
    -novnc					erase -vnc parameters while launching emulators
    -mesa                               use mesa launch emulators default use GPU
    -bridge                             use user-defined bridge0 default use docker0
    -sdkapi <level>                     add -sdkapi parameters while launching emulators
    
e.g: 
   $0 -d p31
   $0 -a 0-19
   $0 -r 0-19
   $0 -r 1,3,5,7-11
    "
}

parseRange()
{
    for extra in ${@:2}
    do
    	checkExtraFlag ${extra}
    done

    checkXorgConfig
    #GPU_num=$?
    echo "GPU number is ${GPU_num} now."

    if [ ${novnc} -eq 1 ];then
    	echo -e "${RED_COLOR} No vnc provided while launching emulator! ${COLOR_END}"
    fi

    range=${1:-0-15}
    tmp_str=${range//,/}
    tmp_str=${tmp_str//-/}
    tmp_str=$(grep '^[[:digit:]]*$' <<< "${tmp_str}")
    if [ "${tmp_str}" = "" ];then
    	range="0-15"
    fi
    range=$(rangeN $range |sort -un)
    range=$(echo -e ${range[*]})
    if [ ${debug} -eq 1 ];then
    	echo -e "op range in $range"
    fi


}

checkExtraFlag()
{
    local extra_flag=$1
    local extra_first_flag=${extra_flag%=*}
    local extra_second_flag=${extra_flag#*=}
    case ${extra_first_flag} in
    	-novnc)		novnc=1;;
    	-showkernel)	showkernel=1;;
    	-wipedata)	wipedata=1;;
    	-async)		async=1;;
        -mesa)          mesa=1;;
        -bridge)        userdefinedbridge=1;;
        -docker)        userdefinedbridge=0;;
	-sdkapi)        sdkapi=$extra_second_flag;;
    	 *)		:;;
    esac
}

#############################
#
# main()
#
#############################

# This script does not support running on KVM virtual machine
if [ ! -c /dev/kvm ];then
   echo "Error: missing /dev/kvm,are you in KVM guest OS?"
   exit 1
fi

case $1 in
    -d|-deploy)  		checkExtraFlag $3;setEnv $2;;
    -u|-upgrade)			upgradeEnv $2;;
    -c|-clean)   			clean;; 
    -cc)   					dockerClean;; 
    -r|-run|-l|-lauch)   	parseRange ${@:2};restartMN ${range};;
    -a|-avd)         		parseRange ${@:2};createAvdMN ${range};;
    -rmavd)					removeAvd;;
    -restart|-reboot)   	parseRange ${@:2};restartMN ${range};;
    -stop)   				parseRange ${@:2};stopMN ${range};;
    -start)   				parseRange ${@:2};startMN ${range};;
    -rd)                    parseRange ${@:2};restartMN_debug ${range};;
    -s|-set)    			parseRange ${@:2};startTime=$(date +%s);emuSetMN ${range};;
    -e)         			parseRange ${@:2};relaunchEmuMN ${range};;
    -relaunch)				parseRange ${@:2};relaunchEmuDocker ${range};;
    -chk)         			parseRange ${@:2};checkReadyMN ${range};;
    -od)                                parseRange ${@:2};launchDocker ${range};;
    -v|-version)			version;;
    -h|-help)   			_help;; 
    *)          			_help;;
esac
